﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace WI_E2_Parser
{
    class Program
    {
        List<string> ignoreExtensions = new List<string>();

        public void addIgnores()
        {
            ignoreExtensions.Add(".png");
            ignoreExtensions.Add(".css");
            ignoreExtensions.Add(".jpg");
            ignoreExtensions.Add(".jpeg");
            ignoreExtensions.Add(".ico");
            ignoreExtensions.Add(".bmp");
            ignoreExtensions.Add(".js");
        }

        static int CompareByKey(Term a, Term b)
        {
            return a.termName.CompareTo(b.termName);
        }
        
        static void Main(string[] args)
        {
            int starttime = (Int32)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds;
            Indexer index = new Indexer();
            List<DomainName> domainNames = new List<DomainName>();
            List<Sites> sites = new List<Sites>();
            List<string> linksToCrawl = new List<string>();

            Crawler crawler = new Crawler();
            Program p = new Program();
            p.addIgnores();

            DomainName seed1 = new DomainName("http://dk-kogebogen.dk");

            domainNames.Add(seed1);

            linksToCrawl.Add("http://dk-kogebogen.dk");
            int i = 0;
            int crawlCount = 0;
            while (linksToCrawl.Count > 0 && crawlCount <= 100)
            {
                Uri Url = null;
                if (i >= linksToCrawl.Count) {
                    i = 0;
                }
                try
                {
                    Url = new Uri(linksToCrawl[i]);
                }
                catch (UriFormatException e)
                {
                    linksToCrawl.RemoveAt(i);
                    i--;
                }

                string domain = "";
                if(Url != null)
                { 
                    domain = Url.GetLeftPart( UriPartial.Authority );
                }
                    
                DomainName response = domainNames.Find(r => r.domainName == domain.GetHashCode());


                //add domain to domain names
                if (response == null)
                {
                    domainNames.Add(new DomainName(domain));
                }
                else
                { 
                    if (response.allowedToVisit())
                    {
                        if(!index.contains(linksToCrawl[i]))
                        {
                            response.updateTimeStamp();
                            linksToCrawl.AddRange(crawler.findLinks(linksToCrawl[i], p.ignoreExtensions, response.restrictions));
                            index.addPage(linksToCrawl[i], crawler.resultTokens);
                            linksToCrawl.RemoveAt(i);
                            i--;
                            crawlCount++;
                            
                        }
                    }
                }
                i++;
            }
            Console.WriteLine("Crawled " + crawlCount + " in " + ((Int32)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds - starttime) + " seconds");
            int h = 1;
            index.termIndex.Sort(CompareByKey);
            Ranker rank = new Ranker(index, crawlCount);

            rank.rankIndex();

            Console.WriteLine("Crawling done, enter query:");
            string query = Console.ReadLine();
            rank.search(query);
        }
    }
}
